import pandas as pd
import numpy as np

from sklearn import metrics
from sklearn.model_selection import cross_val_score, KFold, StratifiedShuffleSplit, ShuffleSplit

# Pytorch
import torch
from torch.autograd import Variable
import torch.nn as nn 
import torchvision.transforms as transforms

# Visualisation libraries

## Progress Bar
import progressbar

## Text
from colorama import Fore, Back, Style
from IPython.display import Image, display, Markdown, Latex, clear_output

## plotly
from plotly.offline import init_notebook_mode, iplot 
import plotly.graph_objs as go
import plotly.offline as py
from plotly.subplots import make_subplots
import plotly.express as px

## seaborn
import seaborn as sns
sns.set_style("whitegrid")
sns.set_context("paper", rc={"font.size":12,"axes.titlesize":14,"axes.labelsize":12})

## matplotlib
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
from matplotlib.patches import Ellipse, Polygon
import matplotlib.gridspec as gridspec
import matplotlib.colors
from pylab import rcParams
from matplotlib.font_manager import FontProperties
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
plt.style.use('seaborn-whitegrid')
import matplotlib as mpl
mpl.rcParams['figure.figsize'] = (17, 6)
mpl.rcParams['axes.labelsize'] = 14
mpl.rcParams['xtick.labelsize'] = 12
mpl.rcParams['ytick.labelsize'] = 12
mpl.rcParams['text.color'] = 'k'
%matplotlib inline

import warnings
warnings.filterwarnings("ignore")

Home / Image Processing

Sign Language Digits

• Modeling: PyTorch Multi-layer Perceptron (MLP) for Multi-Class Classification
• Modeling: CatBoost Classifier

Sign Language Digits

Sign languages are languages that can include simultaneously employing hand gestures, movement, the orientation of the fingers, arms or body, and facial expressions to convey a speaker's ideas [1]. In this article, we develop a stacking classifier model from various scikit-learn classifiers for the Sign Language Digits dataset from Kaggle. The main purpose of this article is to demonstrate implementation of stacking classifiers.

Dataset

• Image size: 64x64
• Color space: Grayscale
• File format: npy
• Number of classes: 10 (Digits: 0-9)
• Number of participant students: 218
• Number of samples per student: 10

Path = 'Sign_Language_Digits'
# loading the dataset
X = np.load(Path + '/X.npy')
y = np.load(Path + '/Y.npy')

Investigating the data, we can see that y represent numbers 0 to 9 that encoded into dummy variables.

_ , Values = np.nonzero(y[:,:])
Values = pd.Series(Values)
Ind = [Values[Values == i].index.min() for i in Values.unique()]

fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(17, 6), subplot_kw={'xticks': [], 'yticks': []})
_ = fig.suptitle('A Sample of Dataset', fontweight='bold', fontsize = 18)
ax = ax.ravel()

font = FontProperties()
font.set_family('fantasy')

Ind = [Values[Values == i].index.min() for i in Values.unique()]
for i in range(len(ax)):
    _ = ax[i].imshow(X[Ind[i],:,:], cmap = 'gray')
    _ = ax[i].set_title('%i' % np.nonzero(y[Ind[i],:]), fontproperties=font, fontsize = 16)
    _ = ax[i].set_aspect(1)
fig.tight_layout()

y = Values.copy()
del Values

Train and Test sets

def Dist_Plot(y, Plot_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
                        specs=[[{"type": "table"},{"type": "pie"}]])
    
    # Right
    fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                         values= ToSeries(y).value_counts().values, pull=[0.1],
                         textfont=dict(size=Plot_dict['textfont']),
                         marker=dict(colors = Plot_dict['PieColors'],
                         line=dict(color='black', width=1))), row=1, col=2)
    fig.update_traces(hole=.5)
    fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
    
    # Left
    
    ## Table
    Table = y.value_counts().to_frame('Count').reset_index(drop = False)
    Table = Table.rename(columns = {'index':Plot_dict['legend_title']}).sort_values(by = [Plot_dict['legend_title']])
    Table['Percentage'] = np.round(100*(Table['Count']/Table['Count'].sum()),2)

    T = Table.copy()
    T['Percentage'] = T['Percentage'].map(lambda x: '%% %.2f' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    TableColors = Plot_dict['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not Plot_dict['height'] == None:
        fig.update_layout(height = Plot_dict['height'])
    fig.show()
    
Plot_dict = dict(column_widths = [0.3, 0.7],  PieColors = None, textfont = 12,
                 TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Dist_Plot(y, Plot_dict)

StratifiedKFold is a variation of k-fold which returns stratified folds: each set contains approximately the same percentage of samples of each target class as the complete set.

Labels = np.sort(y.unique())

Test_Size = 0.3
sss = StratifiedShuffleSplit(n_splits=1, test_size=Test_Size, random_state=42)
_ = sss.get_n_splits(X, y)
for train_index, test_index in sss.split(X, y):
    # X
    if isinstance(X, pd.DataFrame):
        X_train, X_test = X.loc[train_index], X.loc[test_index]
    else:
        X_train, X_test = X[train_index], X[test_index]
    # y    
    if isinstance(y, pd.Series):
        y_train, y_test = y[train_index], y[test_index]
    else:
        y_train, y_test = y[train_index], y[test_index]
del sss

# Flattening the X sets
def Flat(Inp): return Inp.reshape(Inp.shape[0],Inp.shape[1]*Inp.shape[2])
X_train, X_test = Flat(X_train), Flat(X_test)

def Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict):
    def ToSeries(x):
        if not isinstance(x, pd.Series):
            Out = pd.Series(x)
        else:
            Out = x.copy()
        return Out
    
    fig = make_subplots(rows=1, cols=3, horizontal_spacing = 0.02, column_widths= Plot_dict['column_widths'],
                        specs=[[{"type": "table"},{'type':'domain'}, {'type':'domain'}]])
    
    # Right
    C = 2
    for y in [y_train, y_test]:
        fig.add_trace(go.Pie(labels= np.sort(y.unique()),
                             values= ToSeries(y).value_counts().values, pull=[0.1],
                             textfont=dict(size=Plot_dict['textfont']),
                             marker=dict(colors = Plot_dict['PieColors'],
                             line=dict(color='black', width=1))), row=1, col=C)
        fig.update_traces(hole=.5)
        fig.update_layout(legend=dict(orientation="v"), legend_title_text= Plot_dict['legend_title'])
        C+=1
    
    # Left
    
    # Table
    Table = pd.DataFrame(data={'Set':['X_train','X_test','y_train','y_test'],
                       'Shape':[X_train.shape, X_test.shape, y_train.shape, y_test.shape]}).astype(str)
    T = Table.copy()
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
        
    TableColors = Plot_dict['TableColors']
    fig.add_trace(go.Table(header=dict(values = list(Table.columns), line_color='darkslategray',
                                       fill_color= TableColors[0], align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.2, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color= [TableColors[1], TableColors[1]]),
                                  align=['center', 'center'], font_size=12, height=20)), 1, 1)

    fig.update_layout(title={'text': '<b>' + 'Dataset Distribution' + '<b>', 'x':0.5,
                                     'y':0.90, 'xanchor': 'center', 'yanchor': 'top'})
    if not Plot_dict['height'] == None:
        fig.update_layout(height = Plot_dict['height'])
    fig.show()
    
Plot_dict = dict(column_widths = [0.3, 0.3, 0.3],  PieColors = None, textfont = 12,
                 TableColors = ['Navy','White'], height = 450, legend_title = 'Digits')
Train_Test_Dist(X_train, y_train, X_test, y_test, Plot_dict)

Modeling: Keras Multi-layer Perceptron (MLP) for Image Classifications

A multi-layer perceptron (MLP) is a class of feedforward artificial neural network (ANN). The algorithm at each iteration uses the Cross-Entropy Loss to measure the loss, and then the gradient and the model update is calculated. At the end of this iterative process, we would reach a better level of agreement between test and predicted sets since the error would be lower from that of the first step.

Setting up Tensor Arrays

def Header(Text, L = 100, C = 'Blue', T = 'White'):
    BACK = {'Black': Back.BLACK, 'Red':Back.RED, 'Green':Back.GREEN, 'Yellow': Back.YELLOW, 'Blue': Back.BLUE,
         'Magenta':Back.MAGENTA, 'Cyan': Back.CYAN}
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(BACK[C] + FORE[T] + Style.NORMAL + Text + Style.RESET_ALL + ' ' + FORE[C] +
          Style.NORMAL +  (L- len(Text) - 1)*'=' + Style.RESET_ALL)
    
def Line(L=100, C = 'Blue'):
    FORE = {'Black': Fore.BLACK, 'Red':Fore.RED, 'Green':Fore.GREEN, 'Yellow':Fore.YELLOW, 'Blue':Fore.BLUE,
         'Magenta':Fore.MAGENTA, 'Cyan':Fore.CYAN, 'White': Fore.WHITE}
    print(FORE[C] + Style.NORMAL + L*'=' + Style.RESET_ALL)
    
def Search_List(Key, List): return [s for s in List if Key in s]

def TorchSets(Set):
    # Inut: Arrays
    # GPU Cuda
    if isinstance(Set, (pd.DataFrame, pd.Series)):
        Set = Set.values
    if torch.cuda.is_available():
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor).cuda())
        else:
            Out = Variable(torch.from_numpy(Set).cuda())
    # CPU
    else:
        if Set.ndim==1:
            Out = Variable(torch.from_numpy(Set).type(torch.LongTensor))
        else:
            Out = Variable(torch.from_numpy(Set))
    return Out

# Tensors
X_train_tensor = TorchSets(X_train)
y_train_tensor = TorchSets(y_train)
X_test_tensor = TorchSets(X_test)
y_test_tensor = TorchSets(y_test)
    
Batch_size = 100
iteration_number = int(4e4)

epochs_number = int(iteration_number / (len(X_train) / Batch_size))

# Pytorch train and test sets
Train_set = torch.utils.data.TensorDataset(X_train_tensor, y_train_tensor)
Test_set = torch.utils.data.TensorDataset(X_test_tensor, y_test_tensor)

# data loader
train_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)
test_loader = torch.utils.data.DataLoader(Train_set, batch_size = Batch_size, shuffle = False)

Modeling

# Create MLP_Model
class MLP_Model(nn.Module):
    def __init__(self, input_Size, hidden_Size, output_Size):
        super(MLP_Model, self).__init__()
        # Linear function 1:
        self.fc1 = nn.Linear(input_Size, hidden_Size) 
        # Non-linearity 1
        self.relu1 = nn.ReLU()
        
        # Linear function 2:
        self.fc2 = nn.Linear(hidden_Size, int(hidden_Size/4))
        # Non-linearity 2
        self.tanh2 = nn.Tanh()
        
        # Linear function 3:
        self.fc3 = nn.Linear(int(hidden_Size/4), output_Size)  
    
    def forward(self, x):
        # Linear function 1
        out = self.fc1(x)
        # Non-linearity 1
        out = self.relu1(out)
        
        # Linear function 2
        out = self.fc2(out)
        # Non-linearity 2
        out = self.tanh2(out)
               
        # Linear function 4 (readout)
        out = self.fc3(out)
        return out
    
def Plot_history(history, Table_Rows = 25, yLim = 2):
    fig = make_subplots(rows=1, cols=2, horizontal_spacing = 0.02, column_widths=[0.6, 0.4],
                        specs=[[{"type": "scatter"},{"type": "table"}]])
    # Left
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Loss'].astype(float).values.round(4),
                             line=dict(color='OrangeRed', width= 1.5), name = 'Loss'), 1, 1)
    fig.add_trace(go.Scatter(x= history['Iteration'].values, y= history['Accuracy'].astype(float).values,
                             line=dict(color='MidnightBlue', width= 1.5),  name = 'Accuracy'), 1, 1)
    fig.update_layout(legend=dict(x=0, y=1.1, traceorder='reversed', font_size=12),
                  dragmode='select', plot_bgcolor= 'white', height=600, hovermode='closest',
                  legend_orientation='h')
    fig.update_xaxes(range=[history.Iteration.min(), history.Iteration.max()],
                     showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    fig.update_yaxes(range=[0, yLim], showgrid=True, gridwidth=1, gridcolor='Lightgray',
                     showline=True, linewidth=1, linecolor='Lightgray', mirror=True, row=1, col=1)
    # Right
    ind = np.linspace(0, history.shape[0], Table_Rows, endpoint = False).round(0).astype(int)
    ind = np.append(ind, history.index[-1])
    history = history[history.index.isin(ind)]
    T = history.copy()
    T[['Loss','Accuracy']] = T[['Loss','Accuracy']].applymap(lambda x: '%.4e' % x)
    Temp = []
    for i in T.columns:
        Temp.append(T.loc[:,i].values)
    fig.add_trace(go.Table(header=dict(values = list(history.columns), line_color='darkslategray',
                                       fill_color='Navy', align=['center','center'],
                                       font=dict(color='white', size=12), height=25), columnwidth = [0.4, 0.4, 0.4],
                           cells=dict(values=Temp, line_color='darkslategray',
                                      fill=dict(color=['Lavender', 'white', 'white']),
                                      align=['center', 'center'], font_size=12,height=20)), 1, 2)
    fig.show()

Fitting the model

n_Inst = X_train.shape[1]
n_Att = X_train.shape[1]
    
input_Size, output_Size = n_Inst, len(Labels)
hidden_Size = 256

# model
model = MLP_Model(input_Size, hidden_Size, output_Size)

# GPU
if torch.cuda.is_available():
    model.cuda()

# Cross Entropy Loss 
CEL= nn.CrossEntropyLoss()

# Optimizer 
learning_rate = 1e-2
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# Traning the Model
Count = 0
Loss_list = []
Iteration_list = []
Accuracy_list = []
MSE_list = []
MAE_list = []
Steps = 10

Progress_Bar = progressbar.ProgressBar(maxval= int(iteration_number+iteration_number/10),
                                       widgets=[progressbar.Bar('=', '|', '|'), progressbar.Percentage()])
    
# print('---------------------------------------------------------')
for epoch in range(epochs_number):
    for i, (Xtr, ytr) in enumerate(train_loader):
        
        # Variables
        Xtr = Variable(Xtr.view(-1, n_Att))
        ytr = Variable(ytr)
        
        # Set all gradients to zero
        optimizer.zero_grad()
        
        # Forward
        Out = model(Xtr.float())
        
        # loss
        loss = CEL(Out, ytr.long())
        
        # Backward (Calculating the gradients)
        loss.backward()
        
        # Update parameters
        optimizer.step()
        
        Count += 1
        
        del Xtr, ytr
        
        # Predictions
        if Count % Steps == 0:
            # Calculate Accuracy         
            Correct, Total = 0, 0
            # Predictions
            for Xts, yts in test_loader: 
                Xts = Variable(Xts.view(-1, n_Att))
                
                # Forward
                Out = model(Xts.float())
                
                # The maximum value of Out
                Predicted = torch.max(Out.data, 1)[1]
                
                # Total number of yts
                Total += len(yts)
                
                # Total Correct predictions
                Correct += (Predicted == yts).sum()
            del Xts, yts
            # storing loss and iteration
            Loss_list.append(loss.data)
            Iteration_list.append(Count)
            Accuracy_list.append(Correct / float(Total))
            
        Progress_Bar.update(Count)

Progress_Bar.finish()

history = pd.DataFrame({'Iteration': np.array(Iteration_list),
                      'Loss': np.array([x.cpu().data.numpy() for x in Loss_list]),
                      'Accuracy': np.array([x.cpu().data.numpy() for x in Accuracy_list])})
del Loss_list, Iteration_list, Accuracy_list

|=========================================================================|100%

Model Performance

Plot_history(history, Table_Rows = 18, yLim = 3)

Confusion Matrix

The confusion matrix allows for visualization of the performance of an algorithm. Note that due to the size of data, here we don't provide a Cross-validation evaluation. In general, this type of evaluation is preferred.

def Confusion_Mat(CM_Train, CM_Test, PD, n_splits = 10):
    if n_splits == None:
        Titles = ['Train Set', 'Test Set']
    else:
        Titles = ['Train Set (CV = % i)' % n_splits, 'Test Set (CV = % i)' % n_splits]
    CM = [CM_Train, CM_Test]
    Cmap = ['Greens', 'YlGn','Blues', 'PuBu']
    for i in range(2):
        fig, ax = plt.subplots(1, 2, figsize= PD['FS'])
        fig.suptitle(Titles[i], weight = 'bold', fontsize = 16)
        _ = sns.heatmap(CM[i], annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i], ax = ax[0],
                        linewidths = 0.2, cbar_kws={"shrink": PD['shrink']})
        _ = ax[0].set_title('Confusion Matrix');
        Temp = np.round(CM[i].astype('float') / CM[i].sum(axis=1)[:, np.newaxis], 2)
        _ = sns.heatmap(Temp,
                        annot=True, annot_kws={"size": PD['annot_kws']}, cmap=Cmap[2*i+1], ax = ax[1],
                       linewidths = 0.4, vmin=0, vmax=1, cbar_kws={"shrink": PD['shrink']})
        _ = ax[1].set_title('Normalized Confusion Matrix');

        for a in ax:
            _ = a.set_xlabel('Predicted labels')
            _ = a.set_ylabel('True labels'); 
            _ = a.xaxis.set_ticklabels(Labels)
            _ = a.yaxis.set_ticklabels(Labels)
            _ = a.set_aspect(1)

# Train
y_pred = model(X_train_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Train = pd.DataFrame(metrics.classification_report(y_train, y_pred, target_names=Labels, output_dict=True)).T
CM_Train = metrics.confusion_matrix(y_train, y_pred)
# Test
y_pred = model(X_test_tensor.float())
y_pred = torch.max(y_pred.data, 1)[1]
y_pred = y_pred.cpu().data.numpy()
Reports_Test = pd.DataFrame(metrics.classification_report(y_test, y_pred, target_names=Labels, output_dict=True)).T
CM_Test = metrics.confusion_matrix(y_test, y_pred)

Reports_Train = Reports_Train.reset_index().rename(columns ={'index': 'Train Set'})
Reports_Test = Reports_Test.reset_index().rename(columns ={'index': 'Test Set'})
                                                 
display(Reports_Train.style.hide_index().set_properties(**{'background-color': 'HoneyDew', 'color': 'Black'}).\
        set_properties(subset=['Train Set'], **{'background-color': 'SeaGreen', 'color': 'White'}))
display(Reports_Test.style.hide_index().set_properties(**{'background-color': 'Azure', 'color': 'Black'}).\
        set_properties(subset=['Test Set'], **{'background-color': 'RoyalBlue', 'color': 'White'}))

PD = dict(FS = (20, 8), annot_kws = 10, shrink = .6)
Confusion_Mat(CM_Train, CM_Test, PD = PD, n_splits = None)

Train Set	precision	recall	f1-score	support
0	1.000000	1.000000	1.000000	143.000000
1	1.000000	1.000000	1.000000	143.000000
2	1.000000	1.000000	1.000000	144.000000
3	1.000000	1.000000	1.000000	145.000000
4	1.000000	1.000000	1.000000	144.000000
5	1.000000	1.000000	1.000000	146.000000
6	1.000000	1.000000	1.000000	145.000000
7	1.000000	1.000000	1.000000	144.000000
8	1.000000	1.000000	1.000000	144.000000
9	1.000000	1.000000	1.000000	145.000000
accuracy	1.000000	1.000000	1.000000	1.000000
macro avg	1.000000	1.000000	1.000000	1443.000000
weighted avg	1.000000	1.000000	1.000000	1443.000000

Test Set	precision	recall	f1-score	support
0	0.841270	0.868852	0.854839	61.000000
1	0.931034	0.870968	0.900000	62.000000
2	0.750000	0.774194	0.761905	62.000000
3	0.739726	0.870968	0.800000	62.000000
4	0.952381	0.967742	0.960000	62.000000
5	0.870370	0.758065	0.810345	62.000000
6	0.704918	0.693548	0.699187	62.000000
7	0.912281	0.838710	0.873950	62.000000
8	0.709677	0.709677	0.709677	62.000000
9	0.875000	0.903226	0.888889	62.000000
accuracy	0.825525	0.825525	0.825525	0.825525
macro avg	0.828666	0.825595	0.825879	619.000000
weighted avg	0.828645	0.825525	0.825832	619.000000

---

References

1. Sign language Wikipedia Page
2. Sign Language Digits Dataset
3. Mower, Jeffrey P. "PREP-Mt: predictive RNA editor for plant mitochondrial genes." BMC bioinformatics 6.1 (2005): 1-15.

4. Precision and recall wikipedia page

5. Cross-validation: evaluating estimator performance

---

---